﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Abot.Crawler;
using Newtonsoft.Json;
using FC.Spider.Tools;

namespace FC.Spider.Spiders
{
    /// <summary>
    /// 淘宝网蜘蛛
    /// </summary>
    public class TaoBaoSpider<T> : Spider where T : class
    {
        /// <summary>
        /// 爬取结果集合
        /// </summary>
        public List<T> ResultList { get; set; }

        /// <summary>
        /// 构造函数
        /// </summary>
        /// <param name="url">目标地址</param>
        public TaoBaoSpider(string url = null)
        {
            BaseUrl = url ?? "https://sf.taobao.com/item_list.htm?q=";
        }

        /// <summary>
        /// 页面爬取结束进行操作
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        protected override void crawler_ProcessPageCrawlCompletedAsync(object sender, PageCrawlCompletedArgs e)
        {
            var list = e.CrawledPage.AngleSharpHtmlDocument.QuerySelectorAll("script#sf-item-list-data");
            foreach (var item in list)
            {
                var json = item.InnerHtml.Replace("\n", "");
                T obj = json.ToObject<T>();
                ResultList.Add(obj);
            }
        }
    }
}
